import csv
import json
import math
import os
import socket
import struct
import threading
import operator
from datetime import *

import jieba

doc_dir_path = ''


def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False


def get_stopwords(file_path):
    f = open(file_path, encoding='utf-8')
    words = f.read()
    return set(words.split('\n'))


def get_words(text):
    # 分词
    words = jieba.lcut(text, cut_all=False)
    # 清洗停用词
    stop_words = get_stopwords(r'news\stopwords.txt')
    cleaned_dict = {}
    n = 0
    for item in words:
        item = item.strip().lower()
        if item != '' and not is_number(item) and item not in stop_words:
            n = n + 1
            if item in cleaned_dict:
                cleaned_dict[item] = cleaned_dict[item] + 1
            else:
                cleaned_dict[item] = 1
    return n, cleaned_dict


def fetch_from_csv(term):
    with open(r'news\posting list.csv', 'r', encoding='utf-8') as f:
        csv_read = csv.reader(f)
        for line in csv_read:
            if term == line[0]:
                return [line[1], line[2]]
    return False


def result_by_TFIDF(sentence):
    n, cleaned_dict = get_words(sentence)
    TFIDF_scores = {}
    lenth = os.listdir('news/xml/').__len__()
    for term in cleaned_dict.keys():
        r = fetch_from_csv(term)
        if r is False:
            continue
        df = int(r[0])
        docs = r[1].split('\\')
        for doc in docs:
            doc_id, date_time, tf, ld = doc.split('\t')
            doc_id = int(doc_id)
            ld = int(ld)
            tf = (int(tf)) / ld
            idf = math.log2(lenth / df)
            # tf-idf
            s = tf * idf
            if doc_id in TFIDF_scores:
                TFIDF_scores[doc_id] = TFIDF_scores[doc_id] + s
            else:
                TFIDF_scores[doc_id] = s
    # 排序
    TFIDF_scores = sorted(TFIDF_scores.items(), key=operator.itemgetter(1))
    TFIDF_scores.reverse()
    if len(TFIDF_scores) == 0:
        return 0, []
    else:
        return 1, TFIDF_scores


def result_by_BM25(sentence):
    n, cleaned_dict = get_words(sentence)
    BM25_score = {}
    lenth = os.listdir('news/xml/').__len__()
    k = 1.5  # 平衡项
    b = 0.8  # 平衡项
    avgdl = 122  # 平均文档长度
    N = 924  # 语料库中文档的总数
    for term in cleaned_dict.keys():
        r = fetch_from_csv(term)
        if r is False:
            continue
        print(f'r: {r}')
        n_qi = int(r[0])  # 包含了查询词qi的文档数
        docs = r[1].split('\\')
        docs = docs[:-1]
        for doc in docs:
            doc_id, date_time, tf, dl = doc.split('\t')

            doc_id = int(doc_id)
            dl = int(dl)  # 文档d的长度
            frequence = (int(tf)) / dl  # qi出现在d中的次数 除以 d的长度

            # BM25 score (q,d)

            bm_score = (k + 1) * frequence / (k * (1 - b + b * dl / avgdl) + frequence) * math.log(
                (N - n_qi + 0.5) / n_qi + 0.5)

            if doc_id in BM25_score:
                BM25_score[doc_id] = BM25_score[doc_id] + bm_score
            else:
                BM25_score[doc_id] = bm_score
    # 排序
    BM25_score = sorted(BM25_score.items(), key=operator.itemgetter(1))
    # print(BM25_score)
    BM25_score.reverse()
    if len(BM25_score) == 0:
        return 0, []
    else:
        return 1, BM25_score


def search_mode(sentence):
    return result_by_BM25(sentence)
    # return result_by_TFIDF(sentence)



# 将需要的数据以字典形式打包传递给search函数
def send_file(docid):
    global dir_path
    sock.send(str(len(docid)).encode(encoding='utf-8'))
    sock.recv(10)
    for id in docid:
        filename = ('%s.xml' % id)
        filesize_bytes = os.path.getsize(dir_path + filename)  # 得到文件的大小,字节
        dic = {
            'filename': filename,
            'filesize_bytes': filesize_bytes,
        }
        head_info = json.dumps(dic)  # 将字典转换成字符串
        head_info_len = struct.pack('i', len(head_info))  # 将字符串的长度打包
        #   先将报头转换成字符串(json.dumps), 再将字符串的长度打包
        #   发送报头长度,发送报头内容,最后放实际内容
        #   报头内容包括文件名,文件长度
        #   接收时:先接收4个字节的报头长度
        #   将报头长度解压,得到头部信息的大小,
        #   在接收头部信息, 反序列化(json.loads)
        #   最后接收真实文件
        sock.send(head_info_len)  # 发送head_info的长度
        sock.recv(10)
        sock.send(head_info.encode('utf-8'))
        sock.recv(10)
        #   发送真是信息
        with open(dir_path + filename, 'rb') as f:
            data = f.read()
            sock.sendall(data)
        print("send " + filename)


def searchidlist(key, selected=0):
    flag, id_scores = search_mode(key)
    # flag = 0 或 flag = 1，代表搜索结果是否存在
    # id_scores = [(id, scores)] 以二元组存储文档id和tf-idf得分
    doc_id = [i for i, s in id_scores]
    page = []
    for i in range(1, (len(doc_id) // 10 + 2)):
        page.append(i)
    return flag, page, doc_id


# 读取表单数据，获得doc_ID
def search(sentence):
    try:
        if sentence not in ['']:
            flag, page, doc_id = searchidlist(sentence)
            page_dump = json.dumps(page)
            doc_id_dump = json.dumps(doc_id)
            if flag == 0:
                status = '0'
                sock.send(status.encode(encoding='utf-8'))
                sock.recv(10)
                # 搜索结果不存在的情况
            else:
                status = '1'
                sock.send(status.encode(encoding='utf-8'))
                sock.recv(10)

                sock.send(bytes(page_dump.encode(encoding='utf-8')))
                sock.recv(10)

                sock.send(str(len(doc_id_dump)).encode(encoding='utf-8'))
                sock.recv(10)
                sock.sendall(bytes(doc_id_dump.encode(encoding='utf-8')))
                sock.recv(10)

                send_file(doc_id)
                print('发送成功')
        else:
            status = '-1'
            sock.send(status.encode(encoding='utf-8'))  # 键入为空的情况

    except Exception as e:
        print('search error:%s' % str(e))


def tcplink(sock, addr):
    print('----------------------------------')
    print('Create new Thread')
    # sock.send('Welcome!，欢迎使用虎扑新闻搜索引擎！'.encode(encoding='utf-8'))
    # 发送TCP数据。将string中的数据发送到连接的套接字。
    while True:
        sentence = sock.recv(2048).decode('utf-8')
        print('User is Searching :' + sentence)
        search(sentence)


if __name__ == '__main__':
    global dir_path
    dir_path = 'news/xml/'
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # 创建TCP Socket
    s.bind(('0.0.0.0', 7778))
    # 将socket绑定到二元组（host,port）的形式表示地址
    s.listen(5)
    # 开始监听TCP传入连接
    # 参数backlog指定在拒绝连接之前,操作系统可以挂起的最大连接数量
    # 该值至少为1，大部分应用程序设为5
    print('Waiting for connection...')

    while True:
        sock, addr = s.accept()
        print('Accept new connection from %s:%s...' % addr)
        # 接受TCP连接并返回二元组（sock, address）
        # sock是新的套接字对象，可以用来接收和发送数据
        # address是连接客户端的地址

        t = threading.Thread(target=tcplink, args=(sock, addr))
        t.start()
        # 循环体执行逻辑：
        #  1、s.accept()等待客户端接入
        #  2、新的客户端连入
        #  3、s.accept()接收sock和addr
        #  4、创建和开启一个线程
        #  5、循环体从头执行等待新的客户端接入（1、）
